{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test the neural network policy in the training environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Method definition info(Any"
     ]
    }
   ],
   "source": [
    "using POMDPs, StatsBase, POMDPToolbox, QMDP, RLInterface, AutomotiveDrivingModels, AutoViz, SARSOP, Images, PyCall, Reel, Parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "...) in module Base at util.jl:532 overwritten in module Logging at C:\\Users\\Maxime\\.julia\\v0.6\\Logging\\src\\Logging.jl:115.\n",
      "WARNING: Method definition warn(Any...) in module Base at util.jl:585 overwritten in module Logging at C:\\Users\\Maxime\\.julia\\v0.6\\Logging\\src\\Logging.jl:115"
     ]
    }
   ],
   "source": [
    "include(\"occluded_crosswalk_env.jl\")\n",
    "include(\"helpers.jl\")\n",
    "include(\"pomdp_types.jl\")\n",
    "include(\"constant_pedestrian.jl\")\n",
    "include(\"generative_model.jl\")\n",
    "include(\"render_helpers.jl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ".\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "OCPOMDP(CrosswalkEnv(Roadway, AutomotiveDrivingModels.Lane(LaneTag(2, 1), AutomotiveDrivingModels.CurvePt[CurvePt({25.000, -10.000, 1.571}, 0.000, 0.000, NaN), CurvePt({25.000, 10.000, 1.571}, 20.000, 0.000, NaN)], 6.0, AutomotiveDrivingModels.SpeedLimit(-Inf, Inf), AutomotiveDrivingModels.LaneBoundary(:unknown, :unknown), AutomotiveDrivingModels.LaneBoundary(:unknown, :unknown), AutomotiveDrivingModels.LaneConnection[], AutomotiveDrivingModels.LaneConnection[]), AutomotiveDrivingModels.ConvexPolygon[ConvexPolygon: len 4 (max 4 pts)\n",
       "\tVecE2(15.000, -1.500)\n",
       "\tVecE2(15.000, -4.500)\n",
       "\tVecE2(21.500, -4.500)\n",
       "\tVecE2(21.500, -1.500)\n",
       "], EnvParams(2, 50.0, 3.0, 20.0, 6.0, 5.0, 37.0, 8.0, 100, 0.5, 2.0, 10.0)), AutomotiveDrivingModels.LidarSensor([-1.50796, -1.44513, -1.3823, -1.31947, -1.25664, -1.19381, -1.13097, -1.06814, -1.00531, -0.942478  …  1.00531, 1.06814, 1.13097, 1.19381, 1.25664, 1.31947, 1.3823, 1.44513, 1.50796, 1.5708], [4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316  …  4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 4.06439e-316, 2.00932e-315, 5.92631e-316, 5.92632e-316], [4.06439e-316, 2.0004e-315, 2.0004e-315, 5.60492e-316, 5.60492e-316, 5.60492e-316, 5.60492e-316, 5.60492e-316, 5.60492e-316, 5.60492e-316  …  5.60493e-316, 2.00041e-315, 5.60493e-316, 2.00041e-315, 5.60494e-316, 2.00041e-315, 5.60494e-316, 2.00041e-315, 2.00041e-315, 2.00041e-315], 30.0, ConvexPolygon: len 0 (max 4 pts)\n",
       "), Dict{Int64,AutomotiveDrivingModels.DriverModel}(Pair{Int64,AutomotiveDrivingModels.DriverModel}(1, EgoDriver(OCAction(0.0)))), VehicleDef(CAR, 4.000, 1.800), VehicleDef(PEDESTRIAN, 1.000, 1.000), 2.0, 5.0, -5.0, 37.0, 5.0, 0.5, 0.3, 10, 1.0, 0.5, 0.5, -1.0, 0.0, 1.0, 0.95)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rng = MersenneTwister(1)\n",
    "pomdp = OCPOMDP()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PyObject <module 'dqn.nn_wrapper' from 'C:\\\\Users\\\\Maxime\\\\OneDrive - Leland Stanford Junior University\\\\Research\\\\policy-correction\\\\dqn\\\\nn_wrapper.py'>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@pyimport tensorflow as tf\n",
    "nn_wrapper = pyimport(\"dqn.nn_wrapper\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ../../dqn/corr_21/model.ckpt\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-03 11:48:47.709629: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709663: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE2 instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709687: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE3 instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709697: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709704: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709712: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709718: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations.\n",
      "2017-11-03 11:48:47.709726: W c:\\tf_jenkins\\home\\workspace\\release-win\\m\\windows\\py\\35\\tensorflow\\core\\platform\\cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations.\n",
      "[2017-11-03 11:48:47,870] Restoring parameters from ../../dqn/corr_21/model.ckpt\r\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PyObject <dqn.nn_wrapper.NNWrapper object at 0x000000002A6EFF60>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn = nn_wrapper[:NNWrapper](\"../../dqn/single_3/\", scope=\"multi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "preprocess_o (generic function with 2 methods)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function multi_to_singles(obs, nframes=4)    \n",
    "    n_ped = div(size(obs,1)-2,2)\n",
    "    single_obs = Vector{Array{Float64,3}}(n_ped)\n",
    "    o = reshape(obs, (size(obs, 1), nframes))\n",
    "    for i = 2:n_ped+1\n",
    "        single_o = zeros(4, nframes)\n",
    "        single_o[1,:] = o[1,:]\n",
    "        single_o[2,:] = o[2,:]\n",
    "        single_o[3,:] = o[2*i - 1, :]\n",
    "        single_o[4,:] = o[2*i, :]\n",
    "        single_o = reshape(single_o, (4,1,nframes))\n",
    "        single_obs[i-1] = single_o\n",
    "    end\n",
    "    return single_obs\n",
    "end\n",
    "\n",
    "function get_action(pomdp, policy, o_queue, action_map)\n",
    "    singles = multi_to_singles(o_queue)    \n",
    "#     all_val = zeros(4, length(singles))\n",
    "#     for i = 1:length(singles)\n",
    "#         all_val[:, i] = policy[:value](singles[i])\n",
    "# #         println(singles[i])\n",
    "# #         println(\"\\n\")\n",
    "#     end\n",
    "#     val = minimum(all_val, 2)\n",
    "    val = zeros(4)\n",
    "    for o in singles\n",
    "        val += policy[:value](singles[i])\n",
    "    end\n",
    "    return action_map[argmax(val)]\n",
    "end\n",
    "            \n",
    "# function get_action(pomdp, policy, o_queue, action_map)\n",
    "#     ai = policy[:action](o_queue)\n",
    "#     return action_map[ai+1]\n",
    "# end\n",
    "\n",
    "function efficient_dequeue(queue::Array{Float64,3}, new_elem::Array{Float64,3}, nch::Int64=size(new_elem,3),nqueue::Int64=div(size(queue, 3),size(new_elem, 3)))\n",
    "    # first shift all the old element\n",
    "    queue = circshift(queue, (0,0,-nch))\n",
    "    # enqueue the first one\n",
    "    queue[:,:,(nqueue-1)*nch+1:nqueue*nch] = new_elem\n",
    "    return queue\n",
    "end\n",
    "\n",
    "function preprocess_o(o::OCObs, pomdp::OCPOMDP, queue=nothing)\n",
    "    nframes = 4\n",
    "    o_mat = convert_o(Vector{Float64}, o, pomdp)\n",
    "    o_mat = reshape(o_mat, (size(o_mat,1), 1, 1))\n",
    "    if queue == nothing\n",
    "        w, h, nch = size(o_mat)\n",
    "        o_stacked = repeat(reshape(o_mat, (w, h, nch)), outer=(1,1,nframes))\n",
    "        o_stacked = reshape(o_stacked, (w, h, nch*nframes))\n",
    "        return o_stacked\n",
    "    else\n",
    "        queue = efficient_dequeue(queue, o_mat)\n",
    "    end\n",
    "    return queue\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OCAction(2.0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = initialstate(pomdp, rng)\n",
    "o_init = generate_o(pomdp, s, rng)\n",
    "o_p = preprocess_o(o_init, pomdp)\n",
    "get_action(pomdp, nn, o_p, actions(pomdp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average reward: 0.704; Average # of steps: 17.056; # of crashes: 0; # of time out 8; # of successes 742\n"
     ]
    }
   ],
   "source": [
    "r_avg = 0\n",
    "step_avg = 0\n",
    "crash = 0\n",
    "time_out = 0\n",
    "success = 0\n",
    "n_ep = 1000\n",
    "max_steps = 100\n",
    "saved_states = OCState[]\n",
    "\n",
    "for i=1:n_ep    \n",
    "    saved_states = OCState[]\n",
    "    disc = 1.0\n",
    "    r_total = 0.0    \n",
    "    s = initialstate(pomdp, rng)\n",
    "    a_map = actions(pomdp)\n",
    "    o_init = generate_o(pomdp, s, rng)\n",
    "    o = preprocess_o(o_init, pomdp)\n",
    "    step = 1\n",
    "\n",
    "    while !isterminal(pomdp, s) && step <= max_steps # TODO also check for terminal observation\n",
    "        a = get_action(pomdp, nn, o, a_map)\n",
    "#         println(o, \"\\n\\n\")\n",
    "        sp, o_, r = generate_sor(pomdp, s, a, rng)\n",
    "#         println(o_)\n",
    "\n",
    "        push!(saved_states, sp)\n",
    "        if sp.n > 7\n",
    "           return sp\n",
    "        end\n",
    "        r_total += disc*r\n",
    "\n",
    "        s = sp\n",
    "        op = preprocess_o(o_, pomdp, o)\n",
    "#         println(op)\n",
    "        \n",
    "        o = op\n",
    "\n",
    "#         disc *= discount(pomdp)\n",
    "        step += 1\n",
    "    end\n",
    "    r_avg += r_total\n",
    "    if r_total <= -1.02\n",
    "        crash += 1\n",
    "    elseif step >= 100\n",
    "        time_out += 1\n",
    "#         break\n",
    "    else\n",
    "        success += 1\n",
    "    end\n",
    "    step_avg += step\n",
    "#     println(r_total)\n",
    "end\n",
    "r_avg /= n_ep\n",
    "step_avg /= n_ep\n",
    "\n",
    "println(\"Average reward: $r_avg; Average # of steps: $step_avg; # of crashes: $crash; # of time out $time_out; # of successes $success\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+gAAAJYCAIAAAB+fFtyAAAABmJLR0QA/wD/AP+gvaeTAAAVCklEQVR4nO3dXYycV3nA8TPrWa+NHX8k2HLcGL84hN2CI2+aFesEBRxBWMJHN22pHNpCQRVVhCqEekMl1Avai5ILUEQlblpEK0R7EaIQUhrVDcEpOPFGDvEKO90FHL+WjWMcHGwSxx/7Mb0w0GCvvZ7Nes77zPx+V87OjOc565H3n+Mz76QEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB0jlruAQCqq1arDQ8Pp5QeeuihRqORe5y25fsMcDnquQcAqK677rrri1/8Ykrphrds+t4TI7nHaVu33Tr4yU98LKVUq9UefPDB3OMAVJRwB7io9evXn/vFzW+7ZWLhyrzDtLGb+/vO/eI333AALtSVewAAAGB2wh0AAAIQ7gAAEIBwBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQQD33AABVVxRFURRbh4dyD9LmyrLMPQJApdlxBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIIB67gEAqq4sy5HR8Z27x3IP0rY29/cNburNPQVA1dlxBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIIB67gEAqq4oiqIotg4P5R6kzZVlmXsEgEqz4w4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAA9dwDAFRdWZYjo+M7d4/lHqRtbe7vG9zUm3sKgKqz4w4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAA9dwDAFRdURRFUWwdHso9SJsryzL3CACVZscdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAigfs899+SeAaCiBgYGco/QWQYGBvxUAriY2v79+3PPAFBpRVHkHqEj+AAmgEtzVAYAAAKo79rz49wzAFTU2tXXrF19de4pOsjhoy8ePnos9xQAFVXf8fSzuWcAqKjN/X3CvZUOPv/Czt1juacAqChHZQAAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAOq5BwCourIsR0bHXV/8ytnc3ze4qTf3FABVZ8cdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAOq5BwCouqIoiqLYOjyUe5A2V5Zl7hEAKs2OOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAIQ7AAAEINwBACAA4Q4AAAHUcw8AUHVlWY6Mju/cPZZ7kLa1ub9vcFNv7ikAqs6OOwAABGDHHYB5sOSV6ZUnGktemT7dU/vF8q5fXmVjCGCeCXcA5m7BVGPwmbMDo2eLQ5O1xv9//fnVC3ZvXPj4YM+Znlq+6QDainAHYI7WH5r80wdfWXVs6sKbrj06de1jp24bOfPA+xbvfuvC1s8G0H6EOwBzcdOes3/yzVfqk41L3GfpyemP3n9y9c+ntr1zccsGA2hXziAC0LQ3HpycudqXnr+5Xkvpzu+evuXpMy2aDKB9CXcAmtM90fjoN07OUO21lL55d1qx6MKH/OEjp1Yen+FEDQCXrz6w8YY5PGxicurZnxyYmKzo38K1lHo3XLf0dXP5l1lLy8XSZmRpuZxb2trVV5/7z7Wrr778vy1DLO21/KndtuPkihPTM9w83JfetSF9ajD93ePn3VKfbNz53dP/9gdL5vCkl6/tX5CWdh5Ly8XSZtSCpdXffvNb5vbIFcuWPP7UD+d3mvnSu+G6odtunvPDLS0LS7sYS8vivKWtu3bVumtXXf7DAy2tWSuWLRm8b8cMN9RS+uw7Ukrpr29JXxpJx0+fd/tNeyceeF/jil5kpnNekM2ytCws7WIsbc4clQGgCUsPn1r985n2k4b70sDalFJavih9avDC2+uTjTc/N3GFpwNoZ/Vde348h4edfOX03h8fmPdp5sv4c4dSStesXDaHx1paLpY2I0vL5dzSbux947nTMoePvnj46LHLfGyIpc35T2368Zl+avxmu/2ci2y6rz420wGb+dP2L0hLO4+l5WJpM2rB0uo7nn72ij5BFo2Uxp47lHuKK8LSIrK0iM4tbcWypefC/eDzL+zcPZZ7qPnxGv/U3vbLyRm+eucNv9puP2f5onTPQPr898+711UvX9lwb/sXZO4prghLi8jScnFUBoAmnOmZ6QfH+hXnf+UNyy+81+lFfugAzJ2/QwFowomr5v7u0tfyWACEOwBNOLh2walFc+zvnxQ+rhtg7oQ7AE2YWlDb23v+x6Nejp9eW3/hmgXzPg9A5xDuADRn2zsWTS1oetP9kdtn+ERVAC6ff7UEoDkvXNP12Nt77vifV13tcd+L6f69v3WnHzz/6v/64e8u3Pvm7pZMB9C2hDsATXvk9sVrjk7dOPbrD1Tati9t23exOx9eU//6Xa9r0WTk8/oXp28cO9u7b3LlienlL033nGnknqhpZ7trJ5bVTixbML6hvqe3+8hqh7uoFuEOMIuiKIqi2Do8lHuQivnb6fTZx9K951+s/XzvfdPaf//Q51fMfk6mLMt5mYvWW/PC1ND20/17z+Ye5LVaONFYdayx6tj0m/ZPvP87pw5cV3/4jsX71oslqsJrEYA5WdCVPv/uNHR9+ptH01M/neEO11+d/v72dPfGVHMVyLbVNd344KOn3/nk6Vq87fXZrT80+VdffWnn7/U88L7Fk3UvY/IT7gC8Bre/MY18Io39PH37R2n/8XT0ZFqxKK1fnt5zfRpYK9nb26IzjT+//2TfTyZmv2tkm39wZs0LU1+5e+nLS7yeyUy4A/Ca9b0+9b0+9xC0VK2R/uyB9q/2c4qDk3/59Zf/8eNLJ7q1Ozm5HCQA0LQPPHrqrT/qiGo/Z93hyQ/956ncU9DphDsA0Jw1R6e2PHkm9xSt9rZnzmw4MJl7CjqacAcAmvOB75zqmm7Ht6PO5vf/+1St0YkLpyKEOwDQhBUnpt8y3kGHZF5t/aHJ3zkynXsKOpdwBwCasHF8IvM7NLd9JH2wN9eTbxwLf7l64hLuAEATep/Lfc77lnXpWx9OT/xFlnzPv3w6mHAHAJqw4sRU7hFSStnyfeUJR2XIRrgDAE1Y/lKV3p3Z8ny/6uVGZ74xlyoQ7gBAExaerV62tjDfu6Yb3Q7LkIlwBwDaQtaz79AC9dwDAACd4eM3pe752DG89G/yq3w/mD63PW3bNw9PB5Uh3AGAlvjSnWnpwhY9163r0n99JD15MP3D99PD4y16UrjCHJUBANqUwzO0F+EOALQ1+U67EO4AQAeQ78Qn3AGAjiHficybUwFmUZblyOj4zt1juQdpW5v7+wY3qSiAWQh3AKBjuM4MkQl3AKADSHbiE+4AQFuT7LQL4Q4AtCnJTnsR7gBAS3zqkdQ9H5ez+9KdqWe2gHniYPrc9rRt3zw8HVSGcAcAWuKrz8zP7/OFodRz8VvtstO+hDsA0BYkO+1OuAMATTi7sNZztpF7it/WwmSf7koT6olMvPQAgCacuKp21cu5h/iNlu+yv7S0a7qr1rKng1cT7gBAE44vX3Dd81O5p8h2MOYXy+fj/bUwJ8IdAGjC+Ib6xrGzOSd44mD6whO5rhgzvkE7kY3/awQAmrCntzvzCfehr2W8zuOevoW5nhqEOwDQhOPLu/b2dueeIo9yXf3QtQtyT0HnEu4AQHO+/e7FnfkGzYfvWJx7BDqacAcAmnNk1YLv3nqJz0BqTyM3LXzuDQ64k5NwBwCa9u13Ld775g46MFOuq3/j/a/LPQWdTrgDAE1r1NLX/mjJ/97QEe2+/w31r9y9ZLLeiaeDqBThDgDMxZme2j9/eMljb1/UxufdGyk9eXPPlz+69OUlkon8HNUCAOZouqv28B2Ln+pf+N7tp/v3Zr24+xVw4Lr6t+5Y/Nx6sURVeC0CzKIoiqIotg4P5R6kzZVlmXsE5uhnqxb86x8v+Y93L75xbKJ338TKE9Mrfjndcybz1d7n4OzC2vFlXceXdf1oQ31Pb/fPVrnyI9Ui3AGAeXBsZdf2W3q239JxV5uBlnFgCwAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAdQHNt6QewaAilq7+urcI3SWtauv9lMJ4GJq+/fvzz0DdK4jR44cOXIk9xSkNWvWrFmz5mK3FkXRwlk6lw9gArg0H8AEOR05cmR0dDT3FKSU0iXCHQCqoPaZz3wm9wzQuer1end3d+4pSBMTE5OTkxd+fWBgYGBgwI57a5RluWvXrl27duUeBKCi6vfee2/uGaBzbdmyZcuWLbmnIO3YsWP79u0Xfv3Tn/70wMBAy8fpXDt27LjvvvtyTwFQUa4qAwAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAIQ7AAAE4JNTAWZRluXI6PjO3WO5B2lbm/v7Bjf15p4CoOrsuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABBAPfcAAFVXFEVRFFuHh3IP0ubKssw9AkCl2XEHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAggHruAQCqrizLkdHxnbvHcg/Stjb39w1u6s09BUDV2XEHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAHxyKuRUluX27dtzT0EqyzL3CAAwC+EOOZVlKRkBgMvhqAwAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEIBwBwCAAHwAE8AsiqIoimLr8FDuQdqcDyMDuDQ77gAAEIBwBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAARQzz0AQNWVZTkyOr5z91juQdrW5v6+wU29uacAqDo77gAAEIBwBwCAAIQ7AAAEINwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAARQzz0AQNUVRVEUxdbhodyDtLmyLHOPAFBpdtwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIoJ57AICqK8tyZHR85+6x3IO0rc39fYObenNPAVB1dtwBACAA4Q4AAAEIdwAACEC4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIoJ57AICqK4qiKIqtw0O5B2lzZVnmHgGg0uy4AwBAAMIdAAACEO4AABCAcAcAgACEOwAABCDcAQAgAOEOAAABCHcAAAhAuAMAQADCHQAAAhDuAAAQgHAHAIAAhDsAAAQg3AEAIADhDgAAAQh3AAAIQLgDAEAAwh0AAAIQ7gAAEEA99wAA1XXgwIGUUlmWX/6nf/neEyO5x2lbt906+MlPfCz9+hsOwIxquQcAqK5arTY8PJxSeuihhxqNRu5x2pbvMwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMBv+z8DPkQxcV5m6gAAAABJRU5ErkJggg==",
      "text/plain": [
       "Cairo.CairoSurface{UInt32}(Ptr{Void} @0x000000000b4f2e10, 1000.0, 600.0, #undef)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AutoViz.render(saved_states[end], pomdp.env, cam=FitToContentCamera(0.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using AutoViz, Reel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "include(\"render_helpers.jl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "animate_scenes (generic function with 1 method)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function animate_scenes(scenes::Vector{Scene}, pomdp)\n",
    "    duration =length(scenes)*pomdp.ΔT\n",
    "    fps = Int(1/pomdp.ΔT)\n",
    "    cam = FitToContentCamera(0.)\n",
    "    function render_rec(t, dt)\n",
    "        frame_index = Int(floor(t/dt)) + 1\n",
    "        return AutoViz.render(scenes[frame_index], pomdp.env, cam=cam)\n",
    "    end\n",
    "    return duration, fps, render_rec\n",
    "end\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<video autoplay controls><source src=\"files/reel-17294022033908011931.webm?8572366812512782944\" type=\"video/webm\"></video>"
      ],
      "text/plain": [
       "Reel.Frames{MIME{Symbol(\"image/png\")}}(\"C:\\\\Users\\\\Maxime\\\\AppData\\\\Local\\\\Temp\\\\jl_C45.tmp\", 0x000000000000001d, 2.0, nothing)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "duration, fps, render_states = animate_scenes(saved_states, pomdp)\n",
    "speed_factor = 1\n",
    "film = roll(render_states, fps = fps*speed_factor, duration = duration/speed_factor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isterminal(pomdp, saved_states[end])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.0",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
